﻿' ******************************************************************************
' ** 
' **  Yahoo! Managed
' **  Written by Marius Häusler 2011
' **  It would be pleasant, if you contact me when you are using this code.
' **  Contact: YahooFinanceManaged@gmail.com
' **  Project Home: http://code.google.com/p/yahoo-finance-managed/
' **  
' ******************************************************************************
' **  
' **  Copyright 2011 Marius Häusler
' **  
' **  Licensed under the Apache License, Version 2.0 (the "License");
' **  you may not use this file except in compliance with the License.
' **  You may obtain a copy of the License at
' **  
' **    http://www.apache.org/licenses/LICENSE-2.0
' **  
' **  Unless required by applicable law or agreed to in writing, software
' **  distributed under the License is distributed on an "AS IS" BASIS,
' **  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
' **  See the License for the specific language governing permissions and
' **  limitations under the License.
' ** 
' ******************************************************************************
Imports System.Text.RegularExpressions


Namespace YahooManaged.Finance.NonAPI

    ''' <summary>
    ''' Class for searching for Yahoo IDs
    ''' </summary>
    ''' <remarks></remarks>
    Public Class IDSearchDownload
        Inherits Base.StringDownload

        ''' <summary>
        ''' Raises if an asynchronous search completes
        ''' </summary>
        ''' <param name="sender">The event raising object</param>
        ''' <param name="ea">The event args of the asynchronous download</param>
        ''' <remarks></remarks>
        Public Event AsyncDownloadCompleted(ByVal sender As Base.Download, ByVal ea As IDSearchDownloadCompletedEventArgs)
        ''' <summary>
        ''' Raises if an asynchronous search changed in progress
        ''' </summary>
        ''' <param name="sender">The event raising object</param>
        ''' <param name="ea">The event args of the asynchronous download</param>
        ''' <remarks></remarks>
        Public Event AsyncDownloadChanged(ByVal sender As Base.Download, ByVal ea As IDSearchDownloadChangedEventArgs)

        Private mOptions As New IDSearchOptions
        Private mFinished As Boolean = False

        Public Property Options() As IDSearchOptions
            Get
                Return mOptions
            End Get
            Set(ByVal value As IDSearchOptions)
                mOptions = value
            End Set
        End Property

        ''' <summary>
        ''' Downloads search results for Yahoo IDs by keyword and other options
        ''' </summary>
        ''' <param name="text">The used search text</param>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Function Download(ByVal text As String) As API.IDSearchResponse
            If text.Trim = String.Empty Then Throw New ArgumentNullException("text", "The text is empty.")
            Dim url As String = Me.DownloadURL(text, mOptions, 0)
            Dim resp = MyBase.DownloadString(url)
            Return New API.IDSearchResponse(resp.Connection, Me.ToSearchResults(resp.Result, mOptions.Server))
        End Function

        ''' <summary>
        ''' Starts an asynchronous download of search results for Yahoo IDs by keyword and other options
        ''' </summary>
        ''' <param name="text">The used search text</param>
        ''' <param name="userArgs">Individual user argument</param>
        ''' <remarks></remarks>
        Public Overloads Sub DownloadAsync(ByVal text As String, Optional ByVal userArgs As Object = Nothing)
            If text.Trim = String.Empty Then Throw New ArgumentNullException("text", "The text is empty.")
            mFinished = False
            Me.DownloadAsyncRecursive(New AsyncDownloadRecursiveArgs(userArgs) With {.Text = text.Trim, .Options = New IDSearchOptions(mOptions)})
        End Sub

        Private Sub DownloadAsyncRecursive(ByVal dlArgs As AsyncDownloadRecursiveArgs)
            dlArgs.PagesStarted += 1
            Dim url As String = Me.DownloadURL(dlArgs.Text, dlArgs.Options, dlArgs.PagesStarted)
            MyBase.DownloadStreamAsync(url, dlArgs)
        End Sub

        ''' <summary>
        ''' Default constructor
        ''' </summary>
        ''' <remarks></remarks>
        Public Sub New()
        End Sub



        Private Sub DownloadAsync_Completed(ByVal sender As Base.Download, ByVal e As Base.StringDownloadCompletedEventArgs) Handles MyBase.AsyncStringDownloadCompleted
            If e IsNot Nothing AndAlso e.UserArgs IsNot Nothing AndAlso TypeOf e.UserArgs Is AsyncDownloadRecursiveArgs Then
                Dim searchArgs = DirectCast(e.UserArgs, AsyncDownloadRecursiveArgs)
                If e.Response.Connection.State = Base.ConnectionState.Success Then

                    Dim resultsPerPage As Integer = 20
                    If searchArgs.IsFirst Then
                        searchArgs.MaxResults = Me.ToMaxResults(e.Response.Result) - searchArgs.Options.Start
                    End If

                    Dim maximum As Integer = searchArgs.MaxResults
                    If searchArgs.Options.Count > 0 AndAlso searchArgs.Options.Count <= searchArgs.MaxResults Then
                        maximum = searchArgs.Options.Count
                    End If
                    Dim addedItems As Boolean = False
                    Dim lst() As IDSearchResult = Me.ToSearchResults(e.Response.Result, searchArgs.Options.Server)
                    If maximum <= 0 And lst.Length > 0 Then
                        searchArgs.MaxResults = lst.Length - searchArgs.Options.Start
                        maximum = searchArgs.MaxResults
                    End If
                    Dim responseResults As New List(Of IDSearchResult)

                    If searchArgs.Results.Count + lst.Length <= maximum Then
                        responseResults.AddRange(lst)
                    Else
                        For Each result As IDSearchResult In lst
                            If searchArgs.Results.Count + responseResults.Count >= maximum Then
                                Exit For
                            Else
                                responseResults.Add(result)
                            End If
                        Next
                    End If

                    If responseResults.Count > 0 Then
                        searchArgs.Results.AddRange(responseResults)
                        Me.RaiseChanged(searchArgs, e.Response.Connection, responseResults.ToArray, maximum)
                    End If

                    If searchArgs.IsFirst Then
                        searchArgs.IsFirst = False
                        Dim maxVal As Double = (maximum - searchArgs.Results.Count)
                        Dim rest As Double = maxVal Mod resultsPerPage
                        Dim pages As Integer = CInt((maxVal / resultsPerPage) - ((maxVal / resultsPerPage) Mod 1))
                        If rest > 0 And pages = 0 Then pages += 1
                        Dim number As Integer = Math.Min(4, pages)
                        If number >= 1 Then
                            For i As Integer = 1 To number
                                Me.DownloadAsyncRecursive(searchArgs)
                            Next
                        Else
                            Me.RaiseCompleted(searchArgs, e.Response.Connection, responseResults.ToArray, maximum)
                        End If

                    Else
                        If searchArgs.Results.Count >= maximum Then
                            Me.RaiseCompleted(searchArgs, e.Response.Connection, responseResults.ToArray, maximum)
                        Else
                            Dim m As Integer = (maximum Mod resultsPerPage)
                            If m > 0 Then m = m * -1 + resultsPerPage
                            If (((searchArgs.PagesStarted + 1) * resultsPerPage) - m) < maximum Then
                                Me.DownloadAsyncRecursive(searchArgs)
                            End If
                        End If
                    End If


                Else

                    If Not searchArgs.IsFirst Then
                        Dim resultsPerPage As Integer = 20
                        If Not e.Response.Connection.State = Base.ConnectionState.Canceled Then
                            searchArgs.MaxResults -= resultsPerPage
                        Else
                            searchArgs.MaxResults = searchArgs.Results.Count
                        End If

                    End If

                    If searchArgs.Results.Count >= searchArgs.MaxResults Then
                        Me.RaiseCompleted(searchArgs, e.Response.Connection, New IDSearchResult() {}, -1)
                    End If

                End If

            End If
        End Sub
        Private Function DownloadURL(ByVal text As String, ByVal options As IDSearchOptions, ByVal siteZeroBasedIndex As Integer) As String
            Dim url As New Text.StringBuilder
            url.Append("http://")
            url.Append(mHelper.ServerString(options.Server))
            url.Append("finance.yahoo.com/lookup/")
            Select Case options.Type
                Case SecurityType.Any : url.Append("all")
                Case SecurityType.ETF : url.Append("etfs")
                Case SecurityType.Fund : url.Append("funds")
                Case SecurityType.Future : url.Append("futures")
                Case SecurityType.Index : url.Append("indices")
                Case SecurityType.Stock : url.Append("stocks")
                Case SecurityType.Warrant : url.Append("warrants")
                Case SecurityType.Currency : url.Append("currency")
            End Select
            url.Append("?s=")
            url.Append(Uri.EscapeDataString(text))
            url.Append("&t=")
            If options.Type = SecurityType.Fund Then : url.Append("M"c)
            Else : url.Append(Char.ToUpper(options.Type.ToString()(0)))
            End If
            url.Append("&m=")
            Select Case options.Markets
                Case FinancialMarket.France : url.Append("FR")
                Case FinancialMarket.Germany : url.Append("DR")
                Case FinancialMarket.Spain : url.Append("ES")
                Case FinancialMarket.UK : url.Append("GB")
                Case FinancialMarket.UsAndCanada : url.Append("US")
                Case Else : url.Append("ALL")
            End Select
            url.Append("&r=")
            If options.RankedBy <> SecurityRankProperty.NoRanking Then
                Dim rankNumber As Integer = CInt(options.RankedBy)
                If options.RankingDirection = ListSortDirection.Descending Then rankNumber += 1
                url.Append(rankNumber.ToString)
            End If
            url.Append("&b=")
            url.Append(options.Start + (siteZeroBasedIndex * 20))
            Return url.ToString
        End Function

        Private Function ToSearchResults(ByVal html As String, ByVal srv As Server) As IDSearchResult()
            Dim lst As New List(Of IDSearchResult)

            Dim rowMatches As MatchCollection = Regex.Matches(html.Replace(Microsoft.VisualBasic.ChrW(10), ""), "<tr class=""yui-dt-(even|odd)"">.*?</tr>")
            If rowMatches.Count > 0 Then

                Dim tdReg As New Regex("<td.*?>.*?</td>")
                Dim tdStartReg As New Regex("<td.*?>")
                Dim urlReg As New Regex("<a.*?>")

                For Each row As Match In rowMatches
                    Dim tdMatches As MatchCollection = tdReg.Matches(row.Value.Replace("nowrap", ""))
                    If tdMatches.Count >= 1 Then
                        Dim isValid As Boolean = True
                        Dim name As String = String.Empty
                        Dim id As String = String.Empty
                        Dim category As String = String.Empty
                        Dim exchange As String = String.Empty
                        Dim type As String = String.Empty
                        Dim isin As String = String.Empty

                        For i As Integer = 0 To tdMatches.Count - 1
                            If tdMatches(i).Value.IndexOf("td") > -1 Then
                                Select Case i
                                    Case 0
                                        id = urlReg.Replace(tdStartReg.Replace(tdMatches(i).Value, ""), "").Replace("</a>", "").Replace("</td>", "").Trim
                                    Case 1
                                        name = GetTableCellValue(tdStartReg, tdMatches(i).Value).Replace("&amp;", "&")
                                    Case 2
                                        If srv = Server.USA Or srv = MaasOne.YahooManaged.Server.Canada Then

                                        Else
                                            isin = GetTableCellValue(tdStartReg, tdMatches(i).Value)
                                        End If
                                    Case 3
                                        If srv = Server.USA Or srv = Server.Canada Then
                                            type = GetTableCellValue(tdStartReg, tdMatches(i).Value)
                                        Else

                                        End If
                                    Case 4
                                        If srv = Server.USA Then
                                            category = GetTableCellValue(tdStartReg, tdMatches(i).Value).Replace("<a href="""">", "").Replace("&amp;", "&")
                                        ElseIf srv = Server.Canada Then
                                            exchange = GetTableCellValue(tdStartReg, tdMatches(i).Value)
                                        Else
                                            type = GetTableCellValue(tdStartReg, tdMatches(i).Value)
                                        End If
                                    Case 5
                                        exchange = GetTableCellValue(tdStartReg, tdMatches(i).Value)

                                End Select
                            Else
                                isValid = False
                                Exit For
                            End If

                        Next

                        If isValid Then
                            lst.Add(New IDSearchResult(name, id, type, exchange, category, isin))
                        End If
                    End If
                Next

            End If
            Return lst.ToArray
        End Function
        Private Function GetTableCellValue(ByVal reg As Regex, ByVal html As String) As String
            Return reg.Replace(html, "").Replace("</a>", "").Replace("</td>", "").Trim
        End Function
        Private Function ToMaxResults(ByVal html As String) As Integer
            Dim result As Integer = -1
            Dim m As Match = Regex.Match(html.Replace(Microsoft.VisualBasic.ChrW(10), ""), "<div id=""pagination"">.*?</div>")
            If m.Success Then
                Dim txt As String = m.Value.Substring(0, m.Value.IndexOf(" |")).Replace("<div id=""pagination"">", "")
                If Integer.TryParse(txt.Substring(txt.LastIndexOf(" ") + 1).Replace(",", "").Replace(".", ""), result) Then
                    Return result
                Else
                    Return -1
                End If
            Else
                Return -1
            End If
        End Function

        Private Sub RaiseCompleted(ByVal searchArgs As AsyncDownloadRecursiveArgs, ByVal conn As Base.ConnectionInfo, ByVal lastNewResults() As IDSearchResult, ByVal maxAvailable As Integer)
            If Not searchArgs.CompletedEventAlreadyRaised Then
                mFinished = True
                searchArgs.CompletedEventAlreadyRaised = True
                RaiseEvent AsyncDownloadCompleted(Me, New IDSearchDownloadCompletedEventArgs(searchArgs.UserArgs, New API.IDSearchResponse(conn, searchArgs.Results.ToArray), searchArgs.Options, searchArgs.MaxResults + searchArgs.Options.Start))
            End If
        End Sub
        Private Sub RaiseChanged(ByVal searchArgs As AsyncDownloadRecursiveArgs, ByVal conn As Base.ConnectionInfo, ByVal lastResults() As IDSearchResult, ByVal max As Integer)
            If Not mFinished Then RaiseEvent AsyncDownloadChanged(Me, New IDSearchDownloadChangedEventArgs(searchArgs.UserArgs, New API.IDSearchResponse(conn, searchArgs.Results.ToArray), lastResults, max, searchArgs.Results.Count, searchArgs.Options, searchArgs.MaxResults + searchArgs.Options.Start))
        End Sub

        Private Class AsyncDownloadRecursiveArgs
            Inherits Base.DownloadEventArgs
            Public Text As String = String.Empty
            Public IsFirst As Boolean = True
            Public MaxResults As Integer = 0
            Public Results As New List(Of IDSearchResult)
            Public PagesStarted As Integer = -1
            Public Options As New IDSearchOptions
            Public CompletedEventAlreadyRaised As Boolean = False
            Public Sub New(ByVal userArgs As Object)
                MyBase.New(userArgs)
            End Sub
        End Class
    End Class

    ''' <summary>
    ''' Stores the complete result of an asynchronous ID search
    ''' </summary>
    ''' <remarks></remarks>
    Public Class IDSearchDownloadCompletedEventArgs
        Inherits Base.DownloadCompletedEventArgs

        Private mOptions As IDSearchOptions

        Private mMaximumResultsAvailable As Integer = 0
        Public ReadOnly Property MaximumResultsAvailable() As Integer
            Get
                Return mMaximumResultsAvailable
            End Get
        End Property
        ''' <summary>
        ''' The used search options
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public ReadOnly Property Options() As IDSearchOptions
            Get
                Return mOptions
            End Get
        End Property
        ''' <summary>
        ''' Gets the response with ID search results.
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Overloads ReadOnly Property Response() As API.IDSearchResponse
            Get
                Return DirectCast(MyBase.Response, API.IDSearchResponse)
            End Get
        End Property

        Friend Sub New(ByVal userArgs As Object, ByVal resp As API.IDSearchResponse, ByVal opt As IDSearchOptions, ByVal maxAvailable As Integer)
            MyBase.New(userArgs, resp)
            mOptions = opt
            mMaximumResultsAvailable = maxAvailable
        End Sub

    End Class

    ''' <summary>
    ''' Stores the latest downloaded results of an asynchronous ID search since start or last event raised
    ''' </summary>
    ''' <remarks></remarks>
    Public Class IDSearchDownloadChangedEventArgs
        Inherits Base.DownloadChangedEventArgs

        Private mMaximumResultsAvailable As Integer = 0
        Private mNewResults() As IDSearchResult
        Private mOptions As IDSearchOptions

        ''' <summary>
        ''' The used search options
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public ReadOnly Property Options() As IDSearchOptions
            Get
                Return mOptions
            End Get
        End Property
        Public ReadOnly Property MaximumresultsAvailable() As Integer
            Get
                Return mMaximumResultsAvailable
            End Get
        End Property
        Public ReadOnly Property NewResults() As IDSearchResult()
            Get
                Return mNewResults
            End Get
        End Property
        ''' <summary>
        ''' The received list of ID search results.
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Overloads ReadOnly Property Response() As API.IDSearchResponse
            Get
                Return DirectCast(MyBase.Response, API.IDSearchResponse)
            End Get
        End Property

        Friend Sub New(ByVal userArgs As Object, ByVal resp As API.IDSearchResponse, ByVal newResults() As IDSearchResult, ByVal max As Integer, ByVal down As Integer, ByVal opt As IDSearchOptions, ByVal maxAvailable As Integer)
            MyBase.New(userArgs, resp, max, down)
            mOptions = opt
            mNewResults = newResults
            mMaximumResultsAvailable = maxAvailable
        End Sub

    End Class

    ''' <summary>
    ''' Stores settings for Yahoo ID search
    ''' </summary>
    ''' <remarks></remarks>
    Public Class IDSearchOptions
        Inherits ResultCountOptions

        Private mServer As Server = Server.USA
        Private mType As SecurityType = SecurityType.Any
        Private mMarkets As FinancialMarket = FinancialMarket.AllMarkets
        Private mRankedBy As SecurityRankProperty = SecurityRankProperty.NoRanking
        Private mRankingDirection As System.ComponentModel.ListSortDirection = System.ComponentModel.ListSortDirection.Ascending

        ''' <summary>
        ''' The server source for downloading. Supported servers: Canada, France, Germany, Spain, UK, USA
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Property Server As Server
            Get
                Return mServer
            End Get
            Set(ByVal value As Server)
                mServer = value
            End Set
        End Property
        ''' <summary>
        ''' The search will be limited to a special type or all
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Property Type() As SecurityType
            Get
                Return mType
            End Get
            Set(ByVal value As SecurityType)
                mType = value
            End Set
        End Property
        ''' <summary>
        ''' The search will be limited to a special market or all
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks></remarks>
        Public Property Markets() As FinancialMarket
            Get
                Return mMarkets
            End Get
            Set(ByVal value As FinancialMarket)
                mMarkets = value
            End Set
        End Property
        ''' <summary>
        ''' The ranking property of the result list
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks>Will be ignored if "GermanServer" is True</remarks>
        Public Property RankedBy() As SecurityRankProperty
            Get
                Return mRankedBy
            End Get
            Set(ByVal value As SecurityRankProperty)
                mRankedBy = value
            End Set
        End Property
        ''' <summary>
        ''' The ranking direction of the result list
        ''' </summary>
        ''' <value></value>
        ''' <returns></returns>
        ''' <remarks>Will be ignored if "GermanServer" is True</remarks>
        Public Property RankingDirection() As System.ComponentModel.ListSortDirection
            Get
                Return mRankingDirection
            End Get
            Set(ByVal value As System.ComponentModel.ListSortDirection)
                mRankingDirection = value
            End Set
        End Property

        ''' <summary>
        ''' Default constructor
        ''' </summary>
        ''' <remarks></remarks>
        Public Sub New()
            MyBase.New(0, 20)
        End Sub
        Friend Sub New(ByVal original As IDSearchOptions)
            MyBase.new(original)
            If original IsNot Nothing Then
                With original
                    mServer = .Server
                    mType = .Type
                    mMarkets = .Markets
                    mRankedBy = .RankedBy
                    mRankingDirection = .RankingDirection
                End With
            End If
        End Sub

    End Class


End Namespace
